<?php
$start = microtime(true);
set_time_limit(0);
ini_set('memory_limit', '512M');

//stop words
$ignore = array(
    'a',
    'about',
    'an',
    'and',
    'any',
    'are',
    'as',
    'at',
    'be',
    'been',
    'by',
    'but',
    'for',
    'from',
    'had',
    'has',
    'he',
    'him',
    'his',
    'how',
    'i',
    'if',
    'in',
    'is',
    'it',
    'may',
    'me',
    'o',
    'of',
    'on',
    'or',
    'so',
    'she',
    'shall',
    'that',
    'the',
    'their',
    'then',
    'this',
    'to',
    'too',
    'was',
    'we',
    'what',
    'when',
    'where',
    'which',
    'while',
    'who',
    'why',
    'will',
    'with',
    'you',
    'your',
);

include '.././conf/config.php';
include '.././helper/helper.php';

$allWords = array();
try {
    $db = new PDO("mysql:host=" . $dbConfig['host'] . ";dbname=" . $dbConfig['dbName'] . "", $dbConfig['username'], $dbConfig['password'], array(PDO::ATTR_PERSISTENT => true));
} catch (Exception $exc) {
    echo $exc->getMessage();
    die();
}

$db->query('CREATE TABLE IF NOT EXISTS `search_words` (
  `word` varchar(40) NOT NULL,
  `weight` float NOT NULL,
  PRIMARY KEY (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=utf8;');

$db->query('CREATE TABLE IF NOT EXISTS `search_word_index` (
  `word` varchar(40) NOT NULL,
  `hadith_id` int(11) NOT NULL,
  `collection_name` varchar(45) NOT NULL,
  `weight` float NOT NULL,
  INDEX `word_index` (`word` ASC),
  INDEX `weight_index` (`weight` ASC)
) ENGINE=MyISAM DEFAULT CHARSET=utf8;');



echo 'cleaning old data' . "\n";
echo 'cleaning search_word_index' . "\n";
$db->query('truncate table search_word_index');
echo 'cleaning search_words' . "\n";
$db->query('truncate table search_words');

$allWordSth = $db->prepare('INSERT INTO search_words (word) VALUES (?)');
$wordIndexSth = $db->prepare('INSERT INTO search_word_index (word, hadith_id, collection_name, weight) VALUES (?, ?, ?, ?)');


echo 'processing all hadith' . "\n";

$bukhariQ = $db->query('SELECT h_id, full_text FROM bukhari');
$results['bukhari'] = $bukhariQ->fetchAll(PDO::FETCH_ASSOC);

$muslimQ = $db->query('SELECT h_id, full_text FROM muslim');
$results['muslim'] = $muslimQ->fetchAll(PDO::FETCH_ASSOC);

$ryadusSalehinQ = $db->query('SELECT h_id, full_text FROM ryadus_salehin');
$results['ryadus_salehin'] = $ryadusSalehinQ->fetchAll(PDO::FETCH_ASSOC);

$db->beginTransaction();
foreach ($results as $hadithCol => $hadithColData) {
    echo "\n" . 'processing ' . $hadithCol . "\n";
    foreach ($hadithColData as $result) {
        $wordsInNarration = array();
        $allWordInNarration = array();
        $string = preg_replace('/\'|\"|\.|\,|\(|\)|\:|\n|\!|\;|\?|\d/', '', $result['full_text']);
        $words = explode(' ', $string);

        foreach ($words as $word) {
            $str = strtolower(trim($word));
            $allWordInNarration[] = $str;
            if (!$str || in_array($str, $ignore)) {
                continue;
            }

            if (!in_array($str, $wordsInNarration)) {
                $wordsInNarration[] = $str;
                $allWords[]= $str;
            }

        }


        $totalWords = count($allWordInNarration);
        $wordCount = array_count_values($allWordInNarration);

        foreach ($wordsInNarration as $narrationWord) {
            $weight = ($wordCount[$narrationWord] * 100 / $totalWords) + ($totalWords / 100);
            $insertArr = array($narrationWord, $result['h_id'], $hadithCol, $weight);
            $wordIndexSth->execute($insertArr);
        }

        if ($result['h_id'] % 500 == 0 ) {
            echo '=';
        }
        
    }
}

$db->commit();

echo "\n" . 'writing all words';
$allWords = array_unique($allWords);
$db->beginTransaction();
foreach ($allWords as $allWord) {
    $allWordSth->execute(array($allWord));
}
$db->commit();

//$handle = fopen('debug', 'w');
//fwrite($handle, print_r($allWords, true));
//fclose($handle);

$excTime =microtime(true) - $start;
echo "\n" . 'Exicution time ' . $excTime . '(s)';
echo "\n" . 'done ....' . "\n";